git.openwrt.org Git - feed/packages.git/blob

   1 From 30a779770fe690584456970b602ea16ec3f74ce7 Mon Sep 17 00:00:00 2001
   2 From: Steve Dower <steve.dower@python.org>
   3 Date: Thu, 7 Mar 2019 08:05:31 -0800
   4 Subject: [PATCH] bpo-36216: Add check for characters in netloc that normalize
   5  to separators (GH-12201)
   6
   7 ---
   8  Doc/library/urllib.parse.rst                  | 18 +++++++++++++++
   9  Lib/test/test_urlparse.py                     | 23 +++++++++++++++++++
  10  Lib/urllib/parse.py                           | 17 ++++++++++++++
  11  .../2019-03-06-09-38-40.bpo-36216.6q1m4a.rst  |  3 +++
  12  4 files changed, 61 insertions(+)
  13  create mode 100644 Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
  14
  15 diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
  16 index 0c8f0f607314..b565e1edd321 100644
  17 --- a/Doc/library/urllib.parse.rst
  18 +++ b/Doc/library/urllib.parse.rst
  19 @@ -124,6 +124,11 @@ or on combining URL components into a URL string.
  20     Unmatched square brackets in the :attr:`netloc` attribute will raise a
  21     :exc:`ValueError`.
  22
  23 +   Characters in the :attr:`netloc` attribute that decompose under NFKC
  24 +   normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
  25 +   ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
  26 +   decomposed before parsing, no error will be raised.
  27 +
  28     .. versionchanged:: 3.2
  29        Added IPv6 URL parsing capabilities.
  30
  31 @@ -136,6 +141,10 @@ or on combining URL components into a URL string.
  32        Out-of-range port numbers now raise :exc:`ValueError`, instead of
  33        returning :const:`None`.
  34
  35 +   .. versionchanged:: 3.7.3
  36 +      Characters that affect netloc parsing under NFKC normalization will
  37 +      now raise :exc:`ValueError`.
  38 +
  39
  40  .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None)
  41
  42 @@ -257,10 +266,19 @@ or on combining URL components into a URL string.
  43     Unmatched square brackets in the :attr:`netloc` attribute will raise a
  44     :exc:`ValueError`.
  45
  46 +   Characters in the :attr:`netloc` attribute that decompose under NFKC
  47 +   normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
  48 +   ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
  49 +   decomposed before parsing, no error will be raised.
  50 +
  51     .. versionchanged:: 3.6
  52        Out-of-range port numbers now raise :exc:`ValueError`, instead of
  53        returning :const:`None`.
  54
  55 +   .. versionchanged:: 3.7.3
  56 +      Characters that affect netloc parsing under NFKC normalization will
  57 +      now raise :exc:`ValueError`.
  58 +
  59
  60  .. function:: urlunsplit(parts)
  61
  62 diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
  63 index be50b47603aa..e6638aee2244 100644
  64 --- a/Lib/test/test_urlparse.py
  65 +++ b/Lib/test/test_urlparse.py
  66 @@ -1,3 +1,5 @@
  67 +import sys
  68 +import unicodedata
  69  import unittest
  70  import urllib.parse
  71
  72 @@ -984,6 +986,27 @@ def test_all(self):
  73                  expected.append(name)
  74          self.assertCountEqual(urllib.parse.__all__, expected)
  75
  76 +    def test_urlsplit_normalization(self):
  77 +        # Certain characters should never occur in the netloc,
  78 +        # including under normalization.
  79 +        # Ensure that ALL of them are detected and cause an error
  80 +        illegal_chars = '/:#?@'
  81 +        hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars}
  82 +        denorm_chars = [
  83 +            c for c in map(chr, range(128, sys.maxunicode))
  84 +            if (hex_chars & set(unicodedata.decomposition(c).split()))
  85 +            and c not in illegal_chars
  86 +        ]
  87 +        # Sanity check that we found at least one such character
  88 +        self.assertIn('\u2100', denorm_chars)
  89 +        self.assertIn('\uFF03', denorm_chars)
  90 +
  91 +        for scheme in ["http", "https", "ftp"]:
  92 +            for c in denorm_chars:
  93 +                url = "{}://netloc{}false.netloc/path".format(scheme, c)
  94 +                with self.subTest(url=url, char='{:04X}'.format(ord(c))):
  95 +                    with self.assertRaises(ValueError):
  96 +                        urllib.parse.urlsplit(url)
  97
  98  class Utility_Tests(unittest.TestCase):
  99      """Testcase to test the various utility functions in the urllib."""
 100 diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
 101 index f691ab74f87f..39c5d6a80824 100644
 102 --- a/Lib/urllib/parse.py
 103 +++ b/Lib/urllib/parse.py
 104 @@ -391,6 +391,21 @@ def _splitnetloc(url, start=0):
 105              delim = min(delim, wdelim)     # use earliest delim position
 106      return url[start:delim], url[delim:]   # return (domain, rest)
 107
 108 +def _checknetloc(netloc):
 109 +    if not netloc or netloc.isascii():
 110 +        return
 111 +    # looking for characters like \u2100 that expand to 'a/c'
 112 +    # IDNA uses NFKC equivalence, so normalize for this check
 113 +    import unicodedata
 114 +    netloc2 = unicodedata.normalize('NFKC', netloc)
 115 +    if netloc == netloc2:
 116 +        return
 117 +    _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay
 118 +    for c in '/?#@:':
 119 +        if c in netloc2:
 120 +            raise ValueError("netloc '" + netloc2 + "' contains invalid " +
 121 +                             "characters under NFKC normalization")
 122 +
 123  def urlsplit(url, scheme='', allow_fragments=True):
 124      """Parse a URL into 5 components:
 125      <scheme>://<netloc>/<path>?<query>#<fragment>
 126 @@ -419,6 +434,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
 127                  url, fragment = url.split('#', 1)
 128              if '?' in url:
 129                  url, query = url.split('?', 1)
 130 +            _checknetloc(netloc)
 131              v = SplitResult('http', netloc, url, query, fragment)
 132              _parse_cache[key] = v
 133              return _coerce_result(v)
 134 @@ -442,6 +458,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
 135          url, fragment = url.split('#', 1)
 136      if '?' in url:
 137          url, query = url.split('?', 1)
 138 +    _checknetloc(netloc)
 139      v = SplitResult(scheme, netloc, url, query, fragment)
 140      _parse_cache[key] = v
 141      return _coerce_result(v)
 142 diff --git a/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
 143 new file mode 100644
 144 index 000000000000..5546394157f9
 145 --- /dev/null
 146 +++ b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
 147 @@ -0,0 +1,3 @@
 148 +Changes urlsplit() to raise ValueError when the URL contains characters that
 149 +decompose under IDNA encoding (NFKC-normalization) into characters that
 150 +affect how the URL is parsed.